package com.eascs.spider.kuaixiao.spider;

import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import org.apache.http.impl.client.CloseableHttpClient;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.eascs.spider.kuaixiao.manager.TaskManager;
import com.eascs.spider.kuaixiao.util.MongodbUtil;
import com.eascs.spider.kuaixiao.util.NetUtil;
import com.eascs.spider.kuaixiao.util.RedisUtil;

/**
 * 顺企网url链接,请求，解析，入库
 * @author hzp
 *
 */
public class ShunqiwangCrawler implements Runnable{

	private static Logger logger = LoggerFactory.getLogger(ShunqiwangCrawler.class);
	private String url;
	private CloseableHttpClient client = null;
	private TaskManager manager = null;
	private SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
	public ShunqiwangCrawler(String url) {
		this.url = url;
		manager = TaskManager.getInstance();
		this.client = NetUtil.initHttpClient();
	}
	
	@Override
	public void run(){
		try {
			if(url.contains("http://b2b.11467.com/search")){
				extractO2Ourl();
			}else if(url.contains("http://product.11467.com/")){
				extractProUrl();
			}else if(url.contains("http://www.11467.com/")){
				etl();
			}else{
				logger.info("特殊url暂时不做url提取：{}" , url);
			}
		} catch (Exception e) {
			logger.error("url：{}爬取发生异常" , url ,e);
			manager.addUrl(url);
		}
	}
	
	//根据公司黄页提取O2Ourl
	private void extractO2Ourl(){
		Map<String, String> header = new HashMap<>();
		header.put("Referer", "http://b2b.11467.com");
		header.put("Host", "b2b.11467.com");
		//提取当前页面的url
		String html = NetUtil.loadPageRawTextByGet(url, client, header, null);
		Document document = Jsoup.parse(html);
		Elements elements = document.select("ul.companylist li h4 a");
		Set<String> urlSet = new HashSet<>();
		for(Element e : elements){
			logger.debug("公司名字:{} ,链接:{}" ,e.attr("title") ,e.attr("href"));
			urlSet.add("http:" + e.attr("href"));
		}
		
		//检测是否需要翻页
		int pages = 0;
		Elements as = document.select("div.pages a");
		for(Element a : as){
			if(a.text().contains("尾页")){
				String lastUrl = a.attr("href");
				logger.debug("尾页链接:{}" , lastUrl);
				if(lastUrl.contains("b2b.11467.com/search")&&lastUrl.contains("pn")){
					pages = Integer.valueOf(lastUrl.substring(lastUrl.lastIndexOf("-")+3, lastUrl.indexOf(".htm")));
				}else{
					pages = Integer.valueOf(lastUrl.substring(lastUrl.indexOf("-")+1, lastUrl.indexOf(".htm")));
				}
				logger.debug("总页数：{}" ,pages);
				break;
			}
		}
		
		//翻页请求提取url
		if(pages>0){
			for(int i=2;i<=pages;i++){
				String turnUrl = url.replace(".htm", "-"+i+".htm");
				String turnHtml = NetUtil.loadPageRawTextByGet(turnUrl, client, header, null); 
				Elements turnes = Jsoup.parse(turnHtml).select("ul.companylist li h4 a");
				for(Element e : turnes){
					logger.debug("公司名字：{},链接：http:{}" ,e.attr("title") , e.attr("href"));
					urlSet.add("http:"+e.attr("href"));//第一层去重
				}
			}
		}
		//第二层去重，redis
		for(String urlTemp : urlSet){
			/*if(urlTemp.contains("11467.com/search"))
				continue;*/
			if(!RedisUtil.existShunqiwang(urlTemp)){
				manager.addUrl(urlTemp);
			}
		}
	}
	
	//根据产品供应提取公司url
	private void extractProUrl(){
		Map<String, String> header = new HashMap<>();
		header.put("Referer", "http://product.11467.com");
		header.put("Host", "product.11467.com");
		//提取当前页面的url
		String html = NetUtil.loadPageRawTextByGet(url, client, header, null); 
		Document document = Jsoup.parse(html);
		Elements elements = document.select("li.graybg dd a");
		Set<String> urlSet = new HashSet<>();
		for(Element e : elements){
			logger.debug("公司名字：{},链接：{}" ,e.attr("title") , e.attr("href"));
			urlSet.add("http:"+e.attr("href"));//第一层去重
		}
		
		
		//检测是否需要翻页
		int pages = 0;
		Elements as = document.select("div.pages a");
		for(Element a : as){
			if(a.text().contains("尾页")){
				String lastUrl = a.attr("href");
				logger.debug("尾页链接:{}" , lastUrl);
				pages = Integer.valueOf(lastUrl.substring(lastUrl.indexOf("/pn")+3, lastUrl.length()-1));
				logger.debug("总页数：{}" ,pages);
				break;
			}
		}
		//翻页请求提取url
		if(pages>0){
			for(int i=2;i<=pages;i++){
				String turnUrl = url + "pn" + i +"/";
				String turnHtml = NetUtil.loadPageRawTextByGet(turnUrl, client, header, null); 
				Elements turnes = Jsoup.parse(turnHtml).select("li.graybg dd a");
				for(Element e : turnes){
					logger.debug("公司名字：{},链接：http:{}" ,e.attr("title") , e.attr("href"));
					urlSet.add("http:"+e.attr("href"));//第一层去重
				}
			}
		}
		//第二层去重，redis
		for(String urlTemp : urlSet){
			/*if(urlTemp.contains("11467.com/search"))
				continue;*/
			if(!RedisUtil.existShunqiwang(urlTemp)){
				manager.addUrl(urlTemp);
			}
		}
	}
	
	//提取公司字段信息
	private void etl(){
		Map<String, String> header = new HashMap<>();
		header.put("Referer", "http://www.11467.com");
		header.put("Host", "www.11467.com");
		CloseableHttpClient client = NetUtil.initHttpClient();
		String html = NetUtil.loadPageRawTextByGet(url, client, header, null);
		
		Map<String , Object> data = new HashMap<>();
		Document document = Jsoup.parse(html);
		//1，公司介绍
		String name = document.select("div#logo h1").text();
		String aboutus = document.getElementById("aboutus").text();
		logger.debug("公司介绍:{}" ,aboutus);
		data.put("公司介绍", aboutus);
		data.put("名称", name);
		
		//2，联系方式
		Elements dts = document.select("div#contact dt");
		Elements dds = document.select("div#contact dd");
		Map<String, String> contractMap = new HashMap<>();
		for(int i=0;i<dts.size();i++){
			String nameTemp = dts.get(i).text().replace("：", "");
			String valTemp = null;
			if(nameTemp.contains("QQ"))
				continue;
			if(nameTemp.contains("经理手机号码")){
				String ddHtml = dds.get(i).html();
				valTemp = ddHtml.substring(ddHtml.indexOf("/phone/")+7, ddHtml.indexOf(".jpg")+4);
				valTemp = "http://css.11467.com/phone/" + valTemp;
			}else{
				valTemp = dds.get(i).text();
			}
			contractMap.put(nameTemp, valTemp);
			logger.debug("联系方式内容：{}={}" ,nameTemp , valTemp);
		}
		data.put("联系方式", contractMap);
		
		
		//3，工商信息
		Elements trs = document.select("div#gongshang table tr");
		Map<String, String> businessMap = new HashMap<>();
		for(Element tr : trs){
			Elements tds = tr.select("td");
			if(tds.size()==2){
				String nameTemp = tds.get(0).text().replace("：", "");
				String valTemp = tds.get(1).text();
				logger.debug("工商信息:{}={}" ,nameTemp , valTemp);
				businessMap.put(nameTemp, valTemp);
			}
		}
		data.put("工商信息", businessMap);
		
		//4，产品供应信息
		Elements as = document.select("div[class=box sidesubcat coproduct]").select("li a");
		Map<String, String> productMap = new HashMap<>();
		int count=1;
		for(Element a : as){
			String productName = a.attr("title");
			logger.debug("产品供应信息：{}" ,productName);
			productMap.put("产品供应"+count, productName);
		}
		data.put("产品供应信息", productMap);
		data.put("爬取时间", sdf.format(new Date(System.currentTimeMillis())));
		MongodbUtil.setShunqiwang(data);
		RedisUtil.setShunqiwang(url);
		logger.info("{}爬取成功" ,name);
	}
}
